package com.huang.task;

import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * @author HuangShen
 * @Description
 * @create 2021-04-04 12:15
 */
@Component
public class JobProcess  implements PageProcessor {

    private String url="https://search.51job.com/list/000000,000000,0000,00,9,99,web%25E5%2589%258D%25E7%25AB%25AF,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
    @Override
    public void process(Page page) {

    }

    @Override
    public Site getSite() {
        return null;
    }

    private  Site site=Site.me()
            .setCharset("utf8")
            .setTimeOut(30000)
            .setRetrySleepTime(3000)
            .setRetryTimes(3);



    @Scheduled(initialDelay = 1000,fixedDelay = 10000)
    public  void  process(){
        Spider.create(new JobProcess())
                .addUrl(url)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(10)
                .run();
    }
}
